RMD environment requirements:

#rStudio 3.2.4 #Internet access to download DropBox hosted csv file [128MB] #Folder ‘sfzipcodes’ located in RMD’s working directory #https://github.com/avennu01/three-amigos/tree/master/sfzipcodes

Load and wrangle the data

## dat_crime <- read.csv(unz("https://github.com/avennu01/three-amigos/blob/master/train.csv.zip","train.csv")) .. unable to direclty read data from GitHub

dat_crime <- read.csv("https://www.dropbox.com/s/kjkt5ndf3jkibq4/train.csv?raw=1")

#table(dat_crime$Category)

# seperating Dates in to Date & Time and selecting below categories only, rest have been eliminated based on group consensus since they dint have impact on our end objective

# Missing Person .. Need more deepdive 

dat_crimeNew <- dat_crime %>% separate(col = Dates, into = c("Date","Time"), sep = " ",fill = "right" ) %>% filter(Category %in% c("ASSAULT","DISORDERLY CONDUCT","DRUNKENNESS","DRUG/NARCOTIC","KIDNAPPING","LARCENY/THEFT","LOITERING","MISSING PERSON","NON-CRIMINAL","ROBBERY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUSPICIOUS OCC","VANDALISM","VEHICLE THEFT"))

dat_occ <- dat_crimeNew %>% group_by(Category,Descript) %>% summarize(occurence= n())

# Based of above table, below considerations have been made 
# Descripts to consider apart from one's selected .... ASSAULT, BATTERY,STALKING,LOITERING,
# complete categories to consider "KIDNAPPING","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE"
# complete categories to ingore "VANDALISM","VEHICLE THEFT"

dat_crimeNew <- dat_crime %>% separate(col = Dates, into = c("Date","Time"), sep = " ",fill = "right" ) %>% filter(Category %in% c("ASSAULT","DISORDERLY CONDUCT","DRUNKENNESS","DRUG/NARCOTIC","KIDNAPPING","LARCENY/THEFT","LOITERING","MISSING PERSON","NON-CRIMINAL","ROBBERY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUSPICIOUS OCC"))

dat_occ <- dat_crimeNew %>% group_by(Category,Descript) %>% summarize(occurence= n())

p <- dat_occ %>% spread(Descript,occurence)

final_desc <- c("AGGRAVATED ASSAULT WITH","ASSAULT, AGGRAVATED, W/","ATTEMPTED HOMICIDE WITH","ATTEMPTED MAYHEM WITH","ATTEMPTED SIMPLE","BATTERY WITH","MAYHEM WITH","THREATS AGAINST","WILLFUL CRUELTY","COMMITTING PUBLIC","DISTURBING THE PEAC","MAINTAINING A PUBLIC","FOR SALE","SALE OF","UNDER INFLUENCE","UNDER THE INFLUENCE","ATTEMPTED PETTY THEFT","GRAND THEFT PICK","GRAND THEFT PURSE","PETTY THEFT","THEFT, DRUNK ROLL,","AIDED CASE, DOG","AIDED CASE, INJURED","ASSAULT TO ROB WITH ","ATTEMPTED ROBBERY ON THE STREET","ATTEMPTED ROBBERY WITH","ROBBERY ON THE STREET ","ROBBERY, ARMED WITH A ","ROBBERY, BODILY","ASSAULT BY JUVENILE","SHOOTING BY JUVENILE","ANNOY OR MOLEST","STOLEN CELLULAR PHONE","STOLEN ELECTRONICS","SUSPICIOUS A","SUSPICIOUS OCCU","SUSPICIOUS PER","MISSING")

select_desc <- function(x) { 
                   names(p %>% select(contains(x)))[-1]
      }

dat_crimefinal <- full_join(dat_crimeNew %>% filter(Descript %in% unlist(lapply(final_desc,select_desc))),dat_crimeNew %>% filter(Category %in% c("KIDNAPPING","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE")))

# Reduction in observations
#dim(dat_crime)
#dim(dat_crimeNew)
#dim(dat_crimefinal)
paste("Original raw data set was ", nrow(dat_crime), " observances.")
## [1] "Original raw data set was  878049  observances."
paste("Original raw data set was BLA BLA", nrow(dat_crimeNew), " observances.")
## [1] "Original raw data set was BLA BLA 509681  observances."
paste("Lastly, all data not impacting a pedistrian was removed, leaving", nrow(dat_crimefinal), " observances.")
## [1] "Lastly, all data not impacting a pedistrian was removed, leaving 195711  observances."
# Based on the dat_occ_final we can categorise the level of serverity of crime if we want to show our results in such manner 
dat_occ_final <- dat_crimefinal %>% group_by(Category,Descript) %>% summarize(occurence= n())

cat_life <- dat_crimefinal %>% filter(Category %in% c("ASSAULT","KIDNAPPING","MISSING PERSON","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","SUSPICIOUS OCC"))
cat_prop <- dat_crimefinal %>% filter(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"))
cat_nui <- dat_crimefinal %>% filter(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"))

  ##Clean up vectors no longer required
rm(dat_crime)
rm(dat_crimeNew)
rm(dat_occ)

Visualizations

#Assigning longitude and latitude to x and y
x <- dat_crimefinal$X
y <- dat_crimefinal$Y

#Creating base map of San Francisco using ggmap
map_SF <- get_map(location = "San Francisco", zoom = 12)
map <- ggmap(map_SF)

#Inspiration and ideas from the following sources
#https://rpubs.com/nickbearman/r-google-map-making
#http://www.r-bloggers.com/contour-and-density-layers-with-ggmap/
#https://rpubs.com/hegupta/151080

#Heat map using all data across all categories with zip code boundaries
W <- dat_crimefinal
plot<- ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = W, aes(x = x, y = y)) +
   stat_density2d(data = W, aes(x = x, y = y,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "blue", high = "red") +
   theme(legend.position = "none")+ ggtitle("All crime categories for wrangled data with zip code boundaries")

#Layer zip codes on above mao

#Zip <- readOGR(".","SFZipCodes")
Zip <- readOGR(dsn=path.expand("./sfzipcodes"), layer = "SFZipCodes")
## OGR data source with driver: ESRI Shapefile 
## Source: "./sfzipcodes", layer: "SFZipCodes"
## with 27 features
## It has 11 fields
Zip <- spTransform(Zip, CRS("+proj=longlat +datum=WGS84"))
Zip <- fortify(Zip)

plot + geom_polygon(aes(x=long, y=lat, group=group), fill='grey', size=.2,color='blue', data=Zip, alpha=0.2)

#Heat map across category deemed as property
x1 <- cat_prop$X
y1 <- cat_prop$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_prop, aes(x = x1, y = y1)) +
   stat_density2d(data = cat_prop, aes(x = x1, y = y1,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "green", high = "black") +
   theme(legend.position = "none")+ ggtitle("Category Property Crimes")

#Heat map across category deemed as life
x2 <- cat_life$X
y2 <- cat_life$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_life, aes(x = x2, y = y2)) +
   stat_density2d(data = cat_life, aes(x = x2, y = y2,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "yellow", high = "blue") +
   theme(legend.position = "none")+ ggtitle("Category Life Crimes")

#Heat map across category deemed as nuisance
x3 <- cat_nui$X
y3 <- cat_nui$Y

ggmap(map_SF, extent = "panel", maprange=FALSE) +
   geom_density2d(data = cat_nui, aes(x = x3, y = y3)) +
   stat_density2d(data = cat_nui, aes(x = x3, y = y3,  fill = ..level.., alpha = ..level..),
                  size = 0.01, bins = 16, geom = 'polygon') +
   scale_fill_gradient(low = "yellow", high = "red") +
   theme(legend.position = "none")+ ggtitle("Category Nuisance Crimes")

#Adding column "class" to dataset using nested if statement with mutate

dat <- dat_crimefinal %>% mutate(class = ifelse(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"), "Prop", ifelse(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"), "Nui", "Life")))

dat1 <- dat %>% group_by(Category) %>% summarize(occurence = n())

#Histogram by Category
qplot(Category, data = dat, geom = "bar", fill = Category) +
    ggtitle("Crime Categories in San Francisco") +
    theme(axis.text.x = element_blank())+
    xlab("Category") + 
    ylab("Occurence")

#Histogram by district
qplot(dat$PdDistrict, data = dat, geom = "bar", fill = class) +
    scale_x_discrete(label = abbreviate) +
    ggtitle("Crime by district and class in San Francisco") +
    xlab("District") + 
    ylab("Crimes by class")

#Further analysis of Southern District

#Daily average crime by class in southern district
dat2 <- dat %>% filter(PdDistrict == "SOUTHERN") %>% group_by(DayOfWeek, class) %>% summarize(DailyAvg = mean(n()))

ggplot(dat2, aes(x = dat2$DayOfWeek, y = dat2$DailyAvg, fill = factor(dat2$class) )) + geom_bar(stat="identity") + labs(x = "Day of Week", y = "Daily Average", title = "Daily Avg Crime by class for Southern District")

#Time of Day analysis for Southern District
dat3 <- dat %>% filter(PdDistrict=="SOUTHERN")

time <- as.data.frame(table(dat3$Time),stringsAsFactors = FALSE)
time$Hour <- substr(time$Var1, 1, 2)
TimeOfDay <- ggplot(time, aes(x = Hour, Freq))
TimeOfDay + stat_summary(fun.y = sum, geom = "bar", fill = "grey") +
    labs(x = "Time of Day", y = "Occurrences", title ="Southern District crime timings")

Three Amigos Navigation

##This section is focused on showing the coverage of the crime data points  
  ##Zoomed map of the wrangled crime points for the whole San Francisco area
map_SF <- get_map(location = "San Francisco, CA", zoom = 13)

ggmap(map_SF) +
  ggtitle("All San Francisco crime posing risk to walkers") +  
  geom_jitter(data = dat_crimefinal, aes(X, Y), alpha=0.2, size=0.5, color="red") 

  ##From: http://www.r-bloggers.com/map-biodiversity-records-with-rgbif-and-ggmap-packages-in-r/
####Calculate the three routes and display a simple navigation view

##2 mile walk # North bound walking path # Aquarium of the Bay
walking_from_1 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_1 <- "2 Beach St, San Francisco, CA 94133"  #Aquarium of the Bay
  walking_route_1 <- route(walking_from_1, walking_to_1, structure = 'route', 
                           mode = 'walking', alternative = FALSE)
##2 mile walk # West bound walking path # Painted Ladies
walking_from_2 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_2 <- "Painted Ladies, Steiner Street, San Francisco, CA 94117"  #Painted Ladies
  walking_route_2 <- route(walking_from_2, walking_to_2, structure = 'route', 
                           mode = 'walking', alternative = FALSE)
##~1 mile walk # East bound walking path # Cupid's Span # We'll be walking this route twice.
walking_from_3 <- "125 3rd Street, San Francisco, CA 94103" #St. Regis SF
  walking_to_3 <- "369 The Embarcadero, San Francisco, CA 94105"  #Cupid's Span
  walking_route_3 <- route(walking_from_3, walking_to_3, structure = 'route', 
                           mode = 'walking', alternative = FALSE)

  ##Center the map for the walking paths
map_SF_1 <- get_map(location = "500 California St, San Francisco, CA 94104", zoom = 14)

  ##Create a simple map to show the three walking paths
    ##Create the third route double thick since we're talking the route twice
ggmap(map_SF_1) + 
  ggtitle("Three 2-mile walking paths from St. Regis") +  
  geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="brown", size = 1.5, lineend='round') + 
    geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="gray", size = 1.5, lineend='round') + 
    geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="tan", size = 3, lineend='round')

##Credit # https://rpubs.com/nickbearman/r-google-map-making
  ##Count the number of route steps
  ###Take off a count, as the last line the in vector is not a movement point
walking_route_1_hop_count <- nrow(walking_route_1) - 1
walking_route_2_hop_count <- nrow(walking_route_2) - 1
walking_route_3_hop_count <- nrow(walking_route_3) - 1

walking_route_1_end <- walking_route_1[walking_route_1_hop_count]

  ##Find the Max North
route_1_mNorth <- max(walking_route_1$lat[1:(walking_route_1_hop_count)])
route_2_mNorth <- max(walking_route_2$lat)
route_3_mNorth <- max(walking_route_3$lat)
  ##Find the Max East
route_1_mEast <- max(walking_route_1$lon)
route_2_mEast <- max(walking_route_2$lon)
route_3_mEast <- max(walking_route_3$lon)
  ##Find the Max South
route_1_mSouth <- min(walking_route_1$lat[1:(walking_route_1_hop_count)])
route_2_mSouth <- min(walking_route_2$lat[1:(walking_route_2_hop_count)])
route_3_mSouth <- min(walking_route_3$lat[1:(walking_route_3_hop_count)])
  ##Find the Max West
route_1_mWest <- min(walking_route_1$lon)
route_2_mWest <- min(walking_route_2$lon[1:(walking_route_2_hop_count)])
route_3_mWest <- min(walking_route_3$lon)
  
  #Find the crime points from the wrangled that within the neighborhood of the respective walking path/route.
    ##We use the North, South, East, & West points to constrain the data for the respective paths
      ##The constrained dataset is called the route's neighborhood; the crime impacting a walking path
dat_crimefinal_route_1 <- dat_crimefinal %>% filter(Y <= (route_1_mNorth + 0.0001),Y >= (route_1_mSouth - 0.0001),X >= (route_1_mWest - 0.001), X <= (route_1_mEast + 0.001))
dat_crimefinal_route_2 <- dat_crimefinal %>% filter(Y <= (route_2_mNorth + 0.0001),Y >= (route_2_mSouth - 0.0001),X >= (route_2_mWest - 0.001), X <= (route_2_mEast + 0.001))
dat_crimefinal_route_3 <- dat_crimefinal %>% filter(Y <= (route_3_mNorth + 0.0001),Y >= (route_3_mSouth - 0.0001),X >= (route_3_mWest - 0.001), X <= (route_3_mEast + 0.001))


##Render the three walking paths, end points, and neighborhood crime points
ggmap(map_SF_1)  + 
  
  ggtitle("Three paths and their neighborhood crime") +
      ## Start with mapping the in-scope crime data points
  geom_jitter(data = dat_crimefinal_route_1, aes(X, Y), alpha=0.3, size=1, color="purple") +
  geom_jitter(data = dat_crimefinal_route_2, aes(X, Y), alpha=0.3, size=1, color="blue") +
  geom_jitter(data = dat_crimefinal_route_3, aes(X, Y), alpha=0.3, size=1, color="orange") +

      ##Add in x and y axis labels
  xlab("Longitude")+ylab("Latitude") +
  
    ## Next, lay out the walking paths for the three routes.
  geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="brown", size = 1.5, lineend='round') +
  geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="gray", size = 1.5, lineend='round') +
  geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="tan", size = 1.5, lineend='round') +
  
    ##Dot & label for Starting Point
  geom_point(data=walking_route_1[1,],  color="white", size=6) +
  geom_label(data=walking_route_1[1,], label="St. Regis", hjust=1.2, vjust=-0.5, size=5 ) +
  
    ##Dots & labels for ending points
  geom_point(data=walking_route_1[walking_route_1_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_1[walking_route_1_hop_count,], label="Aquarium of the Bay", vjust=-1, size=5)+
  geom_point(data=walking_route_2[walking_route_2_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_2[walking_route_2_hop_count,], label="Painted Ladies", vjust=1.5, size=5)+
  geom_point(data=walking_route_3[walking_route_3_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_3[walking_route_3_hop_count,], label="Cupid's Span", vjust=-0.5, size=5)

  ##Combine all three individual routes into a single data frame
dat_crimefinal_route_1 <- dat_crimefinal_route_1 %>% mutate(path = 1)
dat_crimefinal_route_2 <- dat_crimefinal_route_2 %>% mutate(path = 2)
dat_crimefinal_route_3 <- dat_crimefinal_route_3 %>% mutate(path = 3)
dat_crimefinal_routes <- rbind(dat_crimefinal_route_1, dat_crimefinal_route_2, dat_crimefinal_route_3)

 dat_crimefinal_routes <- dat_crimefinal_routes %>% mutate(class = ifelse(Category %in% c("STOLEN PROPERTY","ROBBERY","LARCENY/THEFT"), "Prop", ifelse(Category %in% c("DISORDERLY CONDUCT","DRUG/NARCOTIC","DRUNKENNESS","NON-CRIMINAL"), "Nui", "Life")))

## We wanted to factor the categorized crime higher by a factor of 6, the threat to property by a factor of 3, and nuisance as our baseline.
  # Crime categories = life[*0.6], nuisance[(0.1], property[*0.3]
##Calculating the RISK for path ONE
route_1_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==1, class == "Life") %>% nrow() * 0.6
route_1_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==1, class == "Nui") %>% nrow() * 0.3
route_1_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==1, class == "Prop") %>% nrow() * 0.1
route_1_crime_relative_risk <- route_1_crime_ct_Life + route_1_crime_ct_Nui + route_1_crime_ct_Prop
##Calculating the RISK for path TWO
route_2_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==2, class == "Life") %>% nrow() * 0.6
route_2_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==2, class == "Nui") %>% nrow() * 0.3
route_2_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==2, class == "Prop") %>% nrow() * 0.1
route_2_crime_relative_risk <- route_2_crime_ct_Life + route_2_crime_ct_Nui + route_2_crime_ct_Prop
##Calculating the RISK for path THREE
  #Notice that the crime occurances are duplicated since we're walking this path twice
route_3_crime_ct_Life <- dat_crimefinal_routes %>% filter(path ==2, class == "Life") %>% nrow() * 2 *0.6
route_3_crime_ct_Nui <- dat_crimefinal_routes %>% filter(path ==2, class == "Nui") %>% nrow() * 2* 0.3
route_3_crime_ct_Prop <- dat_crimefinal_routes %>% filter(path ==2, class == "Prop") %>% nrow() *2* 0.1
route_3_crime_relative_risk <- route_3_crime_ct_Life + route_3_crime_ct_Nui + route_3_crime_ct_Prop

  ##Route risk ranking, 10 being worst risk
  ##Make this dynamic in the map rendering
route_risk_rank_1 <- c(1, round(route_1_crime_relative_risk/   max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank_2 <- c(2, round(route_2_crime_relative_risk/ max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank_3 <- c(3, round(route_3_crime_relative_risk / max(route_1_crime_relative_risk,route_2_crime_relative_risk,route_3_crime_relative_risk) * 10))
route_risk_rank <- rbind(route_risk_rank_1, route_risk_rank_2, route_risk_rank_3)
colnames(route_risk_rank) <- c("path", "risk_rank")
data.frame(route_risk_rank)
##                   path risk_rank
## route_risk_rank_1    1         1
## route_risk_rank_2    2         5
## route_risk_rank_3    3        10
##Render a navigation path, with the preferred path emphasized, and relative heatmaps

ggmap(map_SF_1, extent = "panel", maprange=FALSE) +
  ggtitle("Preferred path emphasized and relative heatmaps") +
   ##Add in x and y axis labels
  xlab("Longitude")+ylab("Latitude") +
##Create heatmap for neighborhood #1
   geom_density2d(data = dat_crimefinal_route_1, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_1, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..),
                  size = 0.01, geom = 'polygon') + #, bins = 16
   scale_fill_gradient(low = "blue", high = "red") +
   scale_alpha(range = c(0.00, 0.75), guide = FALSE) +
   theme(legend.position = "none", axis.title = element_blank(), text = element_text(size = 12))+
##Create heatmap for neighborhood #2
  geom_density2d(data = dat_crimefinal_route_2, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_2, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..), size = 0.01, geom = 'polygon') + #, bins = 16
##Create heatmap for neighborhood #3
  geom_density2d(data = dat_crimefinal_route_3, aes(x = X, y = Y)) +
   stat_density2d(data = dat_crimefinal_route_3, aes(x = X, y = Y,  fill = ..level.., alpha = ..level..), size = 0.01, geom = 'polygon') + #, bins = 16
  
  ## Next, lay out the walking paths for the three routes.
      ##Emphasize the route#1 as the preferred path
  geom_path(data = walking_route_1,
        aes(x = lon, y = lat), color="blue", size = 2.5, lineend='round') +
  geom_path(data = walking_route_2,
        aes(x = lon, y = lat), color="orange", size = 1.5, lineend='round') +
  geom_path(data = walking_route_3,
        aes(x = lon, y = lat), color="red", size = 1.5, lineend='round') +
  
  ##Dot & label for Starting Point
  geom_point(data=walking_route_1[1,],  color="white", size=6) +
  geom_label(data=walking_route_1[1,], label="St. Regis", hjust=0.2, vjust=2.0, size=5 ) +
  
    ##Dots & labels for ending points
  geom_point(data=walking_route_1[walking_route_1_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_1[walking_route_1_hop_count,], label="Aquarium of the Bay", vjust=-1, size=5)+
  geom_point(data=walking_route_2[walking_route_2_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_2[walking_route_2_hop_count,], label="Painted Ladies", vjust=1.5, size=5)+
  geom_point(data=walking_route_3[walking_route_3_hop_count,],  color="yellow", size=4) +
    geom_label(data=walking_route_3[walking_route_3_hop_count,], label="Cupid's Span", vjust=-1, size=5)